首页 探索 帮助
注册 登录
J1X
/
huginn
关注 2
点赞 0
派生 0

浏览代码
Basic HuamnTaskAgent with specs

Andrew Cantino 11 年之前
父节点
690eb0f979
当前提交
0120ccb9c1
共有 11 个文件被更改，包括 657 次插入 和 59 次删除
显示文件统计
  1. 28 2
      .env.example
  2. 1 0
      Gemfile
  3. 9 2
      Gemfile.lock
  4. 1 1
      app/assets/javascripts/worker-checker.js.coffee
  5. 1 0
      app/models/agent.rb
  6. 1 7
      app/models/agents/event_formatting_agent.rb
  7. 285 0
      app/models/agents/human_task_agent.rb
  8. 19 0
      lib/utils.rb
  9. 30 0
      spec/lib/utils_spec.rb
  10. 235 0
      spec/models/agents/human_task_agent_spec.rb
  11. 47 47
      spec/models/agents/post_agent_spec.rb

+ 28 - 2
.env.example
查看文件 

@@ -7,7 +7,10 @@ APP_SECRET_TOKEN=REPLACE_ME_NOW!
7 7 
 # for development, but it needs to be changed when you deploy to a production environment.
8 8 
 DOMAIN=localhost:3000
9 9
10 
-# Database Setup
10 
+############################
11 
+#      Database Setup      #
12 
+############################
13 
+
11 14 
 DATABASE_ADAPTER=mysql2
12 15 
 DATABASE_ENCODING=utf8
13 16 
 DATABASE_RECONNECT=true
 
@@ -24,6 +27,10 @@ DATABASE_PASSWORD=""
24 27 
 # Configure Rails environment.  This should only be needed in production and may cause errors in development.
25 28 
 # RAILS_ENV=production
26 29
30 
+#############################
31 
+#    Email Configuration    #
32 
+#############################
33 
+
27 34 
 # Outgoing email settings.  To use Gmail or Google Apps, put your Google Apps domain or gmail.com
28 35 
 # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD.
29 36 
 SMTP_DOMAIN=your-domain-here.com
 
@@ -37,9 +44,28 @@ SMTP_ENABLE_STARTTLS_AUTO=true
37 44 
 # The address from which system emails will appear to be sent.
38 45 
 EMAIL_FROM_ADDRESS=from_address@gmail.com
39 46
47 
+############################
48 
+#     Allowing Signups     #
49 
+############################
50 
+
40 51 
 # This invitation code will be required for users to signup with your Huginn installation.
41 52 
 # You can see its use in user.rb.
42 53 
 INVITATION_CODE=try-huginn
43 54
55 
+###########################
56 
+#      Agent Logging      #
57 
+###########################
58 
+
44 59 
 # Number of lines of log messages to keep per Agent
45 
-AGENT_LOG_LENGTH=100
60 
+AGENT_LOG_LENGTH=200
61 
+
62 
+#############################
63 
+#  AWS and Mechanical Turk  #
64 
+#############################
65 
+
66 
+# AWS Credentials for MTurk
67 
+AWS_ACCESS_KEY_ID="your aws access key id"
68 
+AWS_ACCESS_KEY="your aws access key"
69 
+
70 
+# Set AWS_SANDBOX to true if you're developing Huginn code.
71 
+AWS_SANDBOX=false

+ 1 - 0
Gemfile
查看文件 

@@ -32,6 +32,7 @@ gem 'kramdown'
32 32 
 gem "typhoeus"
33 33 
 gem 'nokogiri'
34 34 
 gem 'wunderground'
35 
+gem 'rturk'
35 36
36 37 
 gem "twitter"
37 38 
 gem 'twitter-stream', '>=0.1.16'

+ 9 - 2
Gemfile.lock
查看文件 

@@ -74,6 +74,8 @@ GEM
74 74 
       http_parser.rb (>= 0.5.3)
75 75 
     em-socksify (0.3.0)
76 76 
       eventmachine (>= 1.0.0.beta.4)
77 
+    erector (0.9.0)
78 
+      treetop (>= 1.2.3)
77 79 
     erubis (2.7.0)
78 80 
     ethon (0.5.12)
79 81 
       ffi (>= 1.3.0)
 
@@ -118,7 +120,7 @@ GEM
118 120 
       mime-types (~> 1.16)
119 121 
       treetop (~> 1.4.8)
120 122 
     method_source (0.8.1)
121 
-    mime-types (1.23)
123 
+    mime-types (1.24)
122 124 
     mini_portile (0.5.1)
123 125 
     multi_json (1.7.9)
124 126 
     multi_xml (0.5.5)
 
@@ -182,6 +184,10 @@ GEM
182 184 
       rspec-core (~> 2.14.0)
183 185 
       rspec-expectations (~> 2.14.0)
184 186 
       rspec-mocks (~> 2.14.0)
187 
+    rturk (2.11.0)
188 
+      erector
189 
+      nokogiri
190 
+      rest-client
185 191 
     rufus-scheduler (2.0.22)
186 192 
       tzinfo (>= 0.3.23)
187 193 
     safe_yaml (0.9.5)
 
@@ -205,7 +211,7 @@ GEM
205 211 
     system_timer (1.2.4)
206 212 
     thor (0.18.1)
207 213 
     tilt (1.4.1)
208 
-    treetop (1.4.14)
214 
+    treetop (1.4.15)
209 215 
       polyglot
210 216 
       polyglot (>= 0.3.1)
211 217 
     twilio-ruby (3.10.0)
 
@@ -269,6 +275,7 @@ DEPENDENCIES
269 275 
   rr
270 276 
   rspec
271 277 
   rspec-rails
278 
+  rturk
272 279 
   rufus-scheduler
273 280 
   sass-rails (~> 3.2.3)
274 281 
   select2-rails

+ 1 - 1
app/assets/javascripts/worker-checker.js.coffee
查看文件 

@@ -8,7 +8,7 @@ $ ->
8 8
9 9 
         if json.pending? && json.pending > 0
10 10 
           tooltipOptions = {
11 
-            title: "#{json.pending} pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures"
11 
+            title: "#{json.pending} jobs pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures"
12 12 
             delay: 0
13 13 
             placement: "bottom"
14 14 
             trigger: "hover"

+ 1 - 0
app/models/agent.rb
查看文件 

@@ -147,6 +147,7 @@ class Agent < ActiveRecord::Base
147 147 
   end
148 148
149 149 
   def log(message, options = {})
150 
+    puts "Agent##{id}: #{message}" unless Rails.env.test?
150 151 
     AgentLog.log_for_agent(self, message, options)
151 152 
   end
152 153

+ 1 - 7
app/models/agents/event_formatting_agent.rb
查看文件 

@@ -66,16 +66,10 @@ module Agents
66 66 
       !recent_error_logs?
67 67 
     end
68 68
69 
-    def value_constructor(value, payload)
70 
-      value.gsub(/<[^>]+>/).each { |jsonpath|
71 
-        Utils.values_at(payload, jsonpath[1..-2]).first.to_s
72 
-      }
73 
-    end
74 
-
75 69 
     def receive(incoming_events)
76 70 
       incoming_events.each do |event|
77 71 
         formatted_event = options[:mode].to_s == "merge" ? event.payload : {}
78 
-        options[:instructions].each_pair {|key, value| formatted_event[key] = value_constructor value, event.payload }
72 
+        options[:instructions].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) }
79 73 
         formatted_event[:agent] = Agent.find(event.agent_id).type.slice!(8..-1) unless options[:skip_agent].to_s == "true"
80 74 
         formatted_event[:created_at] = event.created_at unless options[:skip_created_at].to_s == "true"
81 75 
         create_event :payload => formatted_event

+ 285 - 0
app/models/agents/human_task_agent.rb
查看文件 

@@ -0,0 +1,285 @@
1 
+require 'rturk'
2 
+
3 
+module Agents
4 
+  class HumanTaskAgent < Agent
5 
+    default_schedule "every_10m"
6 
+
7 
+    description <<-MD
8 
+      You can use a HumanTaskAgent to create Human Intelligence Tasks (HITs) on Mechanical Turk.
9 
+
10 
+      HITs can be created in response to events, or on a schedule.  Set `trigger_on` to either `schedule` or `event`.
11 
+
12 
+      The schedule of this Agent is how often it should check for completed HITs, __NOT__ how often to submit one.  To configure how often a new HIT
13 
+      should be submitted when in `schedule` mode, set `submission_period` to a number of hours.
14 
+
15 
+      If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters.
16 
+      For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this:
17 
+
18 
+          {
19 
+            "expected_receive_period_in_days": 2,
20 
+            "trigger_on": "event",
21 
+            "hit": {
22 
+              "max_assignments": 1,
23 
+              "title": "Sentiment evaluation",
24 
+              "description": "Please rate the sentiment of this message: '<$.message>'",
25 
+              "reward": 0.05,
26 
+              "questions": [
27 
+                {
28 
+                  "type": "selection",
29 
+                  "key": "sentiment",
30 
+                  "name": "Sentiment",
31 
+                  "required": "true",
32 
+                  "question": "Please select the best sentiment value:",
33 
+                  "selections": [
34 
+                    { "key": "happy", "text": "Happy" },
35 
+                    { "key": "sad", "text": "Sad" },
36 
+                    { "key": "neutral", "text": "Neutral" }
37 
+                  ]
38 
+                },
39 
+                {
40 
+                  "type": "free_text",
41 
+                  "key": "feedback",
42 
+                  "name": "Have any feedback for us?",
43 
+                  "required": "false",
44 
+                  "question": "Feedback",
45 
+                  "default": "Type here...",
46 
+                  "min_length": "2",
47 
+                  "max_length": "2000"
48 
+                }
49 
+              ]
50 
+            }
51 
+          }
52 
+
53 
+      As you can see, you configure the created HIT with the `hit` option.  Required fields are `title`, which is the
54 
+      title of the created HIT, `description`, which is the description of the HIT, and `questions` which is an array of
55 
+      questions.  Questions can be of `type` _selection_ or _free\\_text_.  Both types require the `key`, `name`, `required`,
56 
+      `type`, and `question` configuration options.  Additionally, _selection_ requires a `selections` array of options, each of
57 
+      which contain `key` and `text`.  For _free\\_text_, the special configuration options are all optional, and are
58 
+      `default`, `min_length`, and `max_length`.
59 
+
60 
+      If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to
61 
+      automatically select the majority vote for each question across all `max_assignments`.
62 
+
63 
+      As with most Agents, `expected_receive_period_in_days` is required if `trigger_on` is set to `event`.
64 
+    MD
65 
+
66 
+    event_description <<-MD
67 
+      Events look like:
68 
+
69 
+          {
70 
+          }
71 
+    MD
72 
+
73 
+    def validate_options
74 
+      errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on])
75 
+
76 
+      if options[:trigger_on] == "event"
77 
+        errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present?
78 
+      elsif options[:trigger_on] == "schedule"
79 
+        errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0
80 
+      end
81 
+
82 
+      if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" }
83 
+        errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
84 
+      end
85 
+    end
86 
+
87 
+    def default_options
88 
+      {
89 
+        :expected_receive_period_in_days => 2,
90 
+        :trigger_on => "event",
91 
+        :hit =>
92 
+          {
93 
+            :max_assignments => 1,
94 
+            :title => "Sentiment evaluation",
95 
+            :description => "Please rate the sentiment of this message: '<$.message>'",
96 
+            :reward => 0.05,
97 
+            :questions =>
98 
+              [
99 
+                {
100 
+                  :type => "selection",
101 
+                  :key => "sentiment",
102 
+                  :name => "Sentiment",
103 
+                  :required => "true",
104 
+                  :question => "Please select the best sentiment value:",
105 
+                  :selections =>
106 
+                    [
107 
+                      { :key => "happy", :text => "Happy" },
108 
+                      { :key => "sad", :text => "Sad" },
109 
+                      { :key => "neutral", :text => "Neutral" }
110 
+                    ]
111 
+                },
112 
+                {
113 
+                  :type => "free_text",
114 
+                  :key => "feedback",
115 
+                  :name => "Have any feedback for us?",
116 
+                  :required => "false",
117 
+                  :question => "Feedback",
118 
+                  :default => "Type here...",
119 
+                  :min_length => "2",
120 
+                  :max_length => "2000"
121 
+                }
122 
+              ]
123 
+          }
124 
+      }
125 
+    end
126 
+
127 
+    def working?
128 
+      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
129 
+    end
130 
+
131 
+    def check
132 
+      setup!
133 
+      review_hits
134 
+
135 
+      if options[:trigger_on] == "schedule" && (memory[:last_schedule] || 0) <= Time.now.to_i - options[:submission_period].to_i * 60 * 60
136 
+        memory[:last_schedule] = Time.now.to_i
137 
+        create_hit
138 
+      end
139 
+    end
140 
+
141 
+    def receive(incoming_events)
142 
+      if options[:trigger_on] == "event"
143 
+        setup!
144 
+
145 
+        incoming_events.each do |event|
146 
+          create_hit event
147 
+        end
148 
+      end
149 
+    end
150 
+
151 
+    # To be moved either into an initilizer or a per-agent setting.
152 
+    def setup!
153 
+      RTurk::logger.level = Logger::DEBUG
154 
+      RTurk.setup(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_ACCESS_KEY'], :sandbox => ENV['AWS_SANDBOX'] == "true") unless Rails.env.test?
155 
+    end
156 
+
157 
+    protected
158 
+
159 
+    def review_hits
160 
+      reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
161 
+      my_reviewed_hit_ids = reviewable_hit_ids & (memory[:hits] || {}).keys.map(&:to_s)
162 
+      log "MTurk reports the following HITs [#{reviewable_hit_ids.to_sentence}], of which I own [#{my_reviewed_hit_ids.to_sentence}]"
163 
+      my_reviewed_hit_ids.each do |hit_id|
164 
+        hit = RTurk::Hit.new(hit_id)
165 
+        assignments = hit.assignments
166 
+
167 
+        log "Looking at HIT #{hit_id}.  I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
168 
+        if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
169 
+          if options[:take_majority] == "true"
170 
+            options[:hit][:questions].each do |question|
171 
+              counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo }
172 
+              assignments.each do |assignment|
173 
+                answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
174 
+                answer = answers[question[:key]]
175 
+                counts[answer] += 1
176 
+              end
177 
+            end
178 
+          else
179 
+            event = create_event :payload => { :answers => assignments.map(&:answers) }
180 
+            log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
181 
+          end
182 
+
183 
+          assignments.each(&:approve!)
184 
+
185 
+          memory[:hits].delete(hit_id.to_sym)
186 
+        end
187 
+      end
188 
+    end
189 
+
190 
+    def create_hit(event = nil)
191 
+      payload = event ? event.payload : {}
192 
+      title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip
193 
+      description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip
194 
+      questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload)
195 
+      hit = RTurk::Hit.create(:title => title) do |hit|
196 
+        hit.max_assignments = (options[:hit][:max_assignments] || 1).to_i
197 
+        hit.description = description
198 
+        hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
199 
+        hit.reward = (options[:hit][:reward] || 0.05).to_f
200 
+        #hit.qualifications.add :approval_rate, { :gt => 80 }
201 
+      end
202 
+      memory[:hits] ||= {}
203 
+      memory[:hits][hit.id] = event && event.id
204 
+      log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
205 
+    end
206 
+
207 
+    # RTurk Question Form
208 
+
209 
+    class AgentQuestionForm < RTurk::QuestionForm
210 
+      needs :title, :description, :questions
211 
+
212 
+      def question_form_content
213 
+        Overview do
214 
+          Title do
215 
+            text @title
216 
+          end
217 
+          Text do
218 
+            text @description
219 
+          end
220 
+        end
221 
+
222 
+        @questions.each.with_index do |question, index|
223 
+          Question do
224 
+            QuestionIdentifier do
225 
+              text question[:key] || "question_#{index}"
226 
+            end
227 
+            DisplayName do
228 
+              text question[:name] || "Question ##{index}"
229 
+            end
230 
+            IsRequired do
231 
+              text question[:required] || 'true'
232 
+            end
233 
+            QuestionContent do
234 
+              Text do
235 
+                text question[:question]
236 
+              end
237 
+            end
238 
+            AnswerSpecification do
239 
+              if question[:type] == "selection"
240 
+
241 
+                SelectionAnswer do
242 
+                  StyleSuggestion do
243 
+                    text 'radiobutton'
244 
+                  end
245 
+                  Selections do
246 
+                    question[:selections].each do |selection|
247 
+                      Selection do
248 
+                        SelectionIdentifier do
249 
+                          text selection[:key]
250 
+                        end
251 
+                        Text do
252 
+                          text selection[:text]
253 
+                        end
254 
+                      end
255 
+                    end
256 
+                  end
257 
+                end
258 
+
259 
+              else
260 
+
261 
+                FreeTextAnswer do
262 
+                  if question[:min_length].present? || question[:max_length].present?
263 
+                    Constraints do
264 
+                      lengths = {}
265 
+                      lengths[:minLength] = question[:min_length].to_s if question[:min_length].present?
266 
+                      lengths[:maxLength] = question[:max_length].to_s if question[:max_length].present?
267 
+                      Length lengths
268 
+                    end
269 
+                  end
270 
+
271 
+                  if question[:default].present?
272 
+                    DefaultText do
273 
+                      text question[:default]
274 
+                    end
275 
+                  end
276 
+                end
277 
+
278 
+              end
279 
+            end
280 
+          end
281 
+        end
282 
+      end
283 
+    end
284 
+  end
285 
+end

+ 19 - 0
lib/utils.rb
查看文件 

@@ -32,6 +32,25 @@ module Utils
32 32 
     end
33 33 
   end
34 34
35 
+  def self.interpolate_jsonpaths(value, data)
36 
+    value.gsub(/<[^>]+>/).each { |jsonpath|
37 
+      Utils.values_at(data, jsonpath[1..-2]).first.to_s
38 
+    }
39 
+  end
40 
+
41 
+  def self.recursively_interpolate_jsonpaths(struct, data)
42 
+    case struct
43 
+      when Hash
44 
+        struct.inject({}) {|memo, (key, value)| memo[key] = recursively_interpolate_jsonpaths(value, data); memo }
45 
+      when Array
46 
+        struct.map {|elem| recursively_interpolate_jsonpaths(elem, data) }
47 
+      when String
48 
+        interpolate_jsonpaths(struct, data)
49 
+      else
50 
+        struct
51 
+    end
52 
+  end
53 
+
35 54 
   def self.value_at(data, path)
36 55 
     values_at(data, path).first
37 56 
   end

+ 30 - 0
spec/lib/utils_spec.rb
查看文件 

@@ -27,6 +27,36 @@ describe Utils do
27 27 
     end
28 28 
   end
29 29
30 
+  describe "#interpolate_jsonpaths" do
31 
+    it "interpolates jsonpath expressions between matching <>'s" do
32 
+      Utils.interpolate_jsonpaths("hello <$.there.world> this <escape works>", { :there => { :world => "WORLD" }, :works => "should work" }).should == "hello WORLD this should+work"
33 
+    end
34 
+  end
35 
+
36 
+  describe "#recursively_interpolate_jsonpaths" do
37 
+    it "interpolates all string values in a structure" do
38 
+      struct = {
39 
+        :int => 5,
40 
+        :string => "this <escape $.works>",
41 
+        :array => ["<works>", "now", "<$.there.world>"],
42 
+        :deep => {
43 
+          :string => "hello <there.world>",
44 
+          :hello => :world
45 
+        }
46 
+      }
47 
+      data = { :there => { :world => "WORLD" }, :works => "should work" }
48 
+      Utils.recursively_interpolate_jsonpaths(struct, data).should == {
49 
+        :int => 5,
50 
+        :string => "this should+work",
51 
+        :array => ["should work", "now", "WORLD"],
52 
+        :deep => {
53 
+          :string => "hello WORLD",
54 
+          :hello => :world
55 
+        }
56 
+      }
57 
+    end
58 
+  end
59 
+
30 60 
   describe "#value_at" do
31 61 
     it "returns the value at a JSON path" do
32 62 
       Utils.value_at({ :foo => { :bar => :baz }}.to_json, "foo.bar").should == "baz"

+ 235 - 0
spec/models/agents/human_task_agent_spec.rb
查看文件 

@@ -0,0 +1,235 @@
1 
+require 'spec_helper'
2 
+
3 
+describe Agents::HumanTaskAgent do
4 
+  before do
5 
+    @checker = Agents::HumanTaskAgent.new(:name => "my human task agent")
6 
+    @checker.options = @checker.default_options
7 
+    @checker.user = users(:bob)
8 
+    @checker.save!
9 
+
10 
+    @event = Event.new
11 
+    @event.agent = agents(:bob_rain_notifier_agent)
12 
+    @event.payload = { :foo => { "bar" => { :baz => "a2b" } },
13 
+                       :name => "Joe" }
14 
+    @event.id = 345
15 
+  end
16 
+
17 
+  describe "when 'trigger_on' is set to 'schedule'" do
18 
+    before do
19 
+      @checker.options[:trigger_on] = "schedule"
20 
+      @checker.options[:submission_period] = "2"
21 
+      @checker.options.delete(:expected_receive_period_in_days)
22 
+    end
23 
+
24 
+    it "should check for reviewable HITs frequently" do
25 
+      mock(@checker).review_hits.twice
26 
+      mock(@checker).create_hit.once
27 
+      @checker.check
28 
+      @checker.check
29 
+    end
30 
+
31 
+    it "should create HITs every 'submission_period' hours" do
32 
+      now = Time.now
33 
+      stub(Time).now { now }
34 
+      mock(@checker).review_hits.times(3)
35 
+      mock(@checker).create_hit.twice
36 
+      @checker.check
37 
+      now += 1 * 60 * 60
38 
+      @checker.check
39 
+      now += 1 * 60 * 60
40 
+      @checker.check
41 
+    end
42 
+
43 
+    it "should ignore events" do
44 
+      mock(@checker).create_hit(anything).times(0)
45 
+      @checker.receive([events(:bob_website_agent_event)])
46 
+    end
47 
+  end
48 
+
49 
+  describe "when 'trigger_on' is set to 'event'" do
50 
+    it "should not create HITs during check but should check for reviewable HITs" do
51 
+      @checker.options[:submission_period] = "2"
52 
+      now = Time.now
53 
+      stub(Time).now { now }
54 
+      mock(@checker).review_hits.times(3)
55 
+      mock(@checker).create_hit.times(0)
56 
+      @checker.check
57 
+      now += 1 * 60 * 60
58 
+      @checker.check
59 
+      now += 1 * 60 * 60
60 
+      @checker.check
61 
+    end
62 
+
63 
+    it "should create HITs based on events" do
64 
+      mock(@checker).create_hit(events(:bob_website_agent_event)).times(1)
65 
+      @checker.receive([events(:bob_website_agent_event)])
66 
+    end
67 
+  end
68 
+
69 
+  describe "creating hits" do
70 
+    it "can create HITs based on events, interpolating their values" do
71 
+      @checker.options[:hit][:title] = "Hi <.name>"
72 
+      @checker.options[:hit][:description] = "Make something for <.name>"
73 
+      @checker.options[:hit][:questions][0][:name] = "<.name> Question 1"
74 
+
75 
+      question_form = nil
76 
+      hitInterface = OpenStruct.new
77 
+      hitInterface.id = 123
78 
+      mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
79 
+      mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }
80 
+
81 
+      @checker.send :create_hit, @event
82 
+
83 
+      hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments]
84 
+      hitInterface.reward.should == @checker.options[:hit][:reward]
85 
+      hitInterface.description.should == "Make something for Joe"
86 
+
87 
+      xml = question_form.to_xml
88 
+      xml.should include("<Title>Hi Joe</Title>")
89 
+      xml.should include("<Text>Make something for Joe</Text>")
90 
+      xml.should include("<DisplayName>Joe Question 1</DisplayName>")
91 
+
92 
+      @checker.memory[:hits][123].should == @event.id
93 
+    end
94 
+
95 
+    it "works without an event too" do
96 
+      @checker.options[:hit][:title] = "Hi <.name>"
97 
+      hitInterface = OpenStruct.new
98 
+      hitInterface.id = 123
99 
+      mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
100 
+      mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
101 
+      @checker.send :create_hit
102 
+      hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments]
103 
+      hitInterface.reward.should == @checker.options[:hit][:reward]
104 
+    end
105 
+  end
106 
+
107 
+  describe "reviewing HITs" do
108 
+    class FakeHit
109 
+      def initialize(options = {})
110 
+        @options = options
111 
+      end
112 
+
113 
+      def assignments
114 
+        @options[:assignments] || []
115 
+      end
116 
+
117 
+      def max_assignments
118 
+        @options[:max_assignments] || 1
119 
+      end
120 
+    end
121 
+
122 
+    class FakeAssignment
123 
+      attr_accessor :approved
124 
+
125 
+      def initialize(options = {})
126 
+        @options = options
127 
+      end
128 
+
129 
+      def answers
130 
+        @options[:answers] || {}
131 
+      end
132 
+
133 
+      def status
134 
+        @options[:status] || ""
135 
+      end
136 
+
137 
+      def approve!
138 
+        @approved = true
139 
+      end
140 
+    end
141 
+
142 
+    it "should work on multiple HITs" do
143 
+      event2 = Event.new
144 
+      event2.agent = agents(:bob_rain_notifier_agent)
145 
+      event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } },
146 
+                          :name2 => "Joe2" }
147 
+      event2.id = 3452
148 
+
149 
+      # It knows about two HITs from two different events.
150 
+      @checker.memory[:hits] = {}
151 
+      @checker.memory[:hits][:"JH3132836336DHG"] = @event.id
152 
+      @checker.memory[:hits][:"JH39AA63836DHG"] = event2.id
153 
+
154 
+      hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
155 
+      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
156 
+
157 
+      # It looksup the two HITs that it owns.  Neither are ready yet.
158 
+      mock(RTurk::Hit).new("JH3132836336DHG") { FakeHit.new }
159 
+      mock(RTurk::Hit).new("JH39AA63836DHG") { FakeHit.new }
160 
+
161 
+      @checker.send :review_hits
162 
+    end
163 
+
164 
+    it "shouldn't do anything if an assignment isn't ready" do
165 
+      @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
166 
+      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
167 
+      assignments = [
168 
+        FakeAssignment.new(:status => "Accepted", :answers => {}),
169 
+        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
170 
+      ]
171 
+      hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
172 
+      mock(RTurk::Hit).new("JH3132836336DHG") { hit }
173 
+
174 
+      # One of the assignments isn't set to "Submitted", so this should get skipped for now.
175 
+      mock.any_instance_of(FakeAssignment).answers.times(0)
176 
+
177 
+      @checker.send :review_hits
178 
+
179 
+      assignments.all? {|a| a.approved == true }.should be_false
180 
+      @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
181 
+    end
182 
+
183 
+    it "shouldn't do anything if an assignment is missing" do
184 
+      @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
185 
+      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
186 
+      assignments = [
187 
+        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
188 
+      ]
189 
+      hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
190 
+      mock(RTurk::Hit).new("JH3132836336DHG") { hit }
191 
+
192 
+      # One of the assignments hasn't shown up yet, so this should get skipped for now.
193 
+      mock.any_instance_of(FakeAssignment).answers.times(0)
194 
+
195 
+      @checker.send :review_hits
196 
+
197 
+      assignments.all? {|a| a.approved == true }.should be_false
198 
+      @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
199 
+    end
200 
+
201 
+    it "should create events when all assignments are ready" do
202 
+      @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
203 
+      mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
204 
+      assignments = [
205 
+        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
206 
+        FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
207 
+      ]
208 
+      hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
209 
+      mock(RTurk::Hit).new("JH3132836336DHG") { hit }
210 
+
211 
+      lambda {
212 
+        @checker.send :review_hits
213 
+      }.should change { Event.count }.by(1)
214 
+
215 
+      assignments.all? {|a| a.approved == true }.should be_true
216 
+
217 
+      @checker.events.last.payload[:answers].should == [
218 
+        {:sentiment => "neutral", :feedback => ""},
219 
+        {:sentiment => "happy", :feedback => "Take 2"}
220 
+      ]
221 
+
222 
+      @checker.memory[:hits].should == {}
223 
+    end
224 
+
225 
+    describe "taking majority votes" do
226 
+      it "should only be valid when all questions are of type 'selection'" do
227 
+
228 
+      end
229 
+
230 
+      it "should take the majority votes of all questions" do
231 
+
232 
+      end
233 
+    end
234 
+  end
235 
+end

+ 47 - 47
spec/models/agents/post_agent_spec.rb
查看文件 

@@ -1,14 +1,14 @@
1 1 
 require 'spec_helper'
2 2
3 3 
 describe Agents::PostAgent do
4 
-    before do
5 
-        @valid_params = {
6 
-            :name => "somename",
7 
-            :options => {
8 
-                :post_url => "http://www.example.com",
9 
-                :expected_receive_period_in_days => 1
10 
-            }
11 
-        }
4 
+  before do
5 
+    @valid_params = {
6 
+      :name => "somename",
7 
+      :options => {
8 
+        :post_url => "http://www.example.com",
9 
+        :expected_receive_period_in_days => 1
10 
+      }
11 
+    }
12 12
13 13 
     @checker = Agents::PostAgent.new(@valid_params)
14 14 
     @checker.user = users(:jane)
 
@@ -17,55 +17,55 @@ describe Agents::PostAgent do
17 17 
     @event = Event.new
18 18 
     @event.agent = agents(:jane_weather_agent)
19 19 
     @event.payload = {
20 
-        :somekey => "somevalue",
21 
-        :someotherkey => {
22 
-            :somekey => "value"
23 
-        }
20 
+      :somekey => "somevalue",
21 
+      :someotherkey => {
22 
+        :somekey => "value"
23 
+      }
24 24 
     }
25 25
26 26 
     @sent_messages = []
27 
-    stub.any_instance_of(Agents::PostAgent).post_event { |uri,event| @sent_messages << event}
28 
-    end
27 
+    stub.any_instance_of(Agents::PostAgent).post_event { |uri, event| @sent_messages << event }
28 
+  end
29 29
30 
-    describe "#receive" do
31 
-        it "checks if it can handle multiple events" do
32 
-            event1 = Event.new
33 
-            event1.agent = agents(:bob_weather_agent)
34 
-            event1.payload = {
35 
-                :xyz => "value1",
36 
-                :message => "value2"
37 
-            }
30 
+  describe "#receive" do
31 
+    it "checks if it can handle multiple events" do
32 
+      event1 = Event.new
33 
+      event1.agent = agents(:bob_weather_agent)
34 
+      event1.payload = {
35 
+        :xyz => "value1",
36 
+        :message => "value2"
37 
+      }
38 38
39 
-            lambda {
40 
-                @checker.receive([@event,event1])
41 
-            }.should change { @sent_messages.length }.by(2)
42 
-        end
39 
+      lambda {
40 
+        @checker.receive([@event, event1])
41 
+      }.should change { @sent_messages.length }.by(2)
43 42 
     end
43 
+  end
44 44
45 
-    describe "#working?" do
46 
-        it "checks if events have been received within expected receive period" do
47 
-            @checker.should_not be_working
48 
-            Agents::PostAgent.async_receive @checker.id, [@event.id]
49 
-            @checker.reload.should be_working
50 
-            two_days_from_now = 2.days.from_now
51 
-            stub(Time).now { two_days_from_now }
52 
-            @checker.reload.should_not be_working
53 
-        end
45 
+  describe "#working?" do
46 
+    it "checks if events have been received within expected receive period" do
47 
+      @checker.should_not be_working
48 
+      Agents::PostAgent.async_receive @checker.id, [@event.id]
49 
+      @checker.reload.should be_working
50 
+      two_days_from_now = 2.days.from_now
51 
+      stub(Time).now { two_days_from_now }
52 
+      @checker.reload.should_not be_working
54 53 
     end
54 
+  end
55 55
56 
-    describe "validation" do
57 
-        before do
58 
-            @checker.should be_valid
59 
-        end
56 
+  describe "validation" do
57 
+    before do
58 
+      @checker.should be_valid
59 
+    end
60 60
61 
-        it "should validate presence of post_url" do
62 
-            @checker.options[:post_url] = ""
63 
-            @checker.should_not be_valid
64 
-        end
61 
+    it "should validate presence of post_url" do
62 
+      @checker.options[:post_url] = ""
63 
+      @checker.should_not be_valid
64 
+    end
65 65
66 
-        it "should validate presence of expected_receive_period_in_days" do
67 
-            @checker.options[:expected_receive_period_in_days] = ""
68 
-            @checker.should_not be_valid
69 
-        end
66 
+    it "should validate presence of expected_receive_period_in_days" do
67 
+      @checker.options[:expected_receive_period_in_days] = ""
68 
+      @checker.should_not be_valid
70 69 
     end
70 
+  end
71 71 
 end